# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
from scrapy.exceptions import IgnoreRequest
import requests
import random
from fake_useragent import UserAgent
from logging import getLogger

logger = getLogger(__name__)
class ProxyMiddleware(object):
    def __init__(self, proxy_pool_url):
        self.proxy_pool_url = proxy_pool_url

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            proxy_pool_url=crawler.settings.get('PROXY_POOL_URL')
        )

    def _get_proxy(self):
        try:
            proxy = requests.get(proxy_pool_url)
            return proxy.text
        except ConnectionError:
            return None

    # 添加代理，需要在request的meta信息中添加proxy字段
    # 代理的形式为: 协议+ip地址+端口
    def process_response(self, request, response, spider):
        if response.status != 200:
            logger.warning('Need use proxy ~~~')
            request.meta["proxy"] = self._get_proxy()
            return request
        else:
            return response

class RandomUserAgentMiddleware(object):

    def process_request(self, request, spider):
        ua = UserAgent(verify_ssl=False)
        request.headers['User-Agent'] = ua.random
        return None